{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file_len = 3328586\n"
     ]
    }
   ],
   "source": [
    "#https://github.com/spro/practical-pytorch/blob/master/char-rnn-generation/char-rnn-generation.ipynb\n",
    "\n",
    "import unidecode\n",
    "import string\n",
    "import random\n",
    "import re\n",
    "\n",
    "all_characters = string.printable\n",
    "n_characters = len(all_characters)\n",
    "\n",
    "file = unidecode.unidecode(open('data/warandpeace.txt').read())\n",
    "file_len = len(file)\n",
    "print('file_len =', file_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "particularly brotherly tenderness, \n",
      "almost lover-like. \n",
      "\n",
      "Owing to the count's customary carelessness \n",
      "nothing was ready for their departure by the \n",
      "twenty-eighth of August and the carts that \n",
      "were to c\n"
     ]
    }
   ],
   "source": [
    "chunk_len = 200\n",
    "\n",
    "def random_chunk():\n",
    "    start_index = random.randint(0, file_len - chunk_len)\n",
    "    end_index = start_index + chunk_len + 1\n",
    "    return file[start_index:end_index]\n",
    "\n",
    "print(random_chunk())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "\n",
    "class RNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size, n_layers=1):\n",
    "        super(RNN, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.n_layers = n_layers\n",
    "        \n",
    "        self.encoder = nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size, n_layers)\n",
    "        self.decoder = nn.Linear(hidden_size, output_size)\n",
    "    \n",
    "    def forward(self, input, hidden):\n",
    "        input = self.encoder(input.view(1, -1))\n",
    "        output, hidden = self.gru(input.view(1, 1, -1), hidden)\n",
    "        output = self.decoder(output.view(1, -1))\n",
    "        return output, hidden\n",
    "\n",
    "    def init_hidden(self):\n",
    "        return torch.zeros(self.n_layers, 1, self.hidden_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 10],\n",
      "        [ 11],\n",
      "        [ 12],\n",
      "        [ 39],\n",
      "        [ 40],\n",
      "        [ 41]])\n"
     ]
    }
   ],
   "source": [
    "# Turn string into list of longs\n",
    "def char_tensor(string):\n",
    "    tensor = torch.zeros(len(string)).long().unsqueeze(1)\n",
    "    for c in range(len(string)):\n",
    "        tensor[c] = all_characters.index(string[c])\n",
    "    return tensor\n",
    "\n",
    "print(char_tensor('abcDEF'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def random_training_set():    \n",
    "    chunk = random_chunk()\n",
    "    inp = char_tensor(chunk[:-1])\n",
    "    target = char_tensor(chunk[1:])\n",
    "    return inp, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(prime_str='A', predict_len=100, temperature=0.8):\n",
    "    hidden = decoder.init_hidden()\n",
    "    prime_input = char_tensor(prime_str)\n",
    "    predicted = prime_str\n",
    "\n",
    "    # Use priming string to \"build up\" hidden state\n",
    "    for p in range(len(prime_str) - 1):\n",
    "        _, hidden = decoder(prime_input[p], hidden)\n",
    "    inp = prime_input[-1]\n",
    "    \n",
    "    for p in range(predict_len):\n",
    "        output, hidden = decoder(inp, hidden)\n",
    "        \n",
    "        # Sample from the network as a multinomial distribution\n",
    "        output_dist = output.data.view(-1).div(temperature).exp()\n",
    "        top_i = torch.multinomial(output_dist, 1)[0]\n",
    "        \n",
    "        \n",
    "        # Add predicted character to string and use as next input\n",
    "        predicted_char = all_characters[top_i]\n",
    "        predicted += predicted_char\n",
    "        inp = char_tensor(predicted_char)\n",
    "\n",
    "    return predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time, math\n",
    "\n",
    "def time_since(since):\n",
    "    s = time.time() - since\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(inp, target):\n",
    "    hidden = decoder.init_hidden()\n",
    "    decoder.zero_grad()\n",
    "    loss = 0\n",
    "\n",
    "    for c in range(chunk_len):\n",
    "        output, hidden = decoder(inp[c], hidden)\n",
    "        loss += criterion(output, target[c])\n",
    "\n",
    "    loss.backward()\n",
    "    decoder_optimizer.step()\n",
    "\n",
    "    return loss.item() / chunk_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0m 16s (100 5%) 2.2805]\n",
      "Lely efisrome tens \n",
      "to sos of ir wor he tho not soburs the contech mats sto sof the tha whe \n",
      "sons decende slol tyiter whe abss the stamed seaplones wafisssk sof ihd he \n",
      "aded atters oy wald an \n",
      "onf the to \n",
      "the \n",
      "hacke son the ferti and tsion he \n",
      "W. \n",
      "Sowas lle of ifler ar the \n",
      "sas to cof berer the Ro saslapblrstet tuptthe mof tulit the \n",
      "pred, haviche eshier lof \n",
      "berers sultise \n",
      "Id wamding Ink ent sente the \n",
      "nerting blomjsee inth- \n",
      "he watresse corich nom he at \n",
      "solve wabi hamt't lit oping \n",
      "\n",
      "\n",
      "\n",
      "wat hat mist thos to hime te the hre \n",
      "rere rermeplise it wemoke \n",
      "artrevtil tho fionthks bar, that at the coll lasl bltilc the \n",
      "\"uly mof lencherse tso fiy the har, te thof laingitdes the Raf molfesser on the wofreme- fi \n",
      "onis of the mof wre lile he \"roysser the thist, beve . the weper, pserrce bovale entard mlofs fener sites \n",
      "efing he- \n",
      "sust to lerey thaker that liblaser the serbe \n",
      "tiend sher thingels thy \n",
      "\n",
      "ofe cof tho thittor thalit hur vovingus nesy \n",
      "at merto wat soshd- \n",
      "\n",
      "ithe wha nas be won let ase to \n",
      "\n",
      "[0m 34s (200 10%) 2.1762]\n",
      "Le dount in avere of ciered and wefaind spe, an of \n",
      "bare whing tha dere wat it in shome wa len of ding the rean be cuportothe \n",
      "as beat the \n",
      "plating reroe an shomew epen inth it the ferov sayfld to ol \n",
      "drigidell verin \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "An'd dior ow tha \n",
      "\n",
      "give tas \n",
      "\n",
      "\n",
      "1ut ir war bemend \n",
      "the bier anr. \n",
      "he of \n",
      "though and \n",
      "the warine sepopcaf wit shov werel ede aid re contirestd is whad of wat the younld of \n",
      "the of frome idel den, whing an- \n",
      "The whe ench and the beas dishuting \n",
      "he and and in the cronored alr. \n",
      "\n",
      "Iff the daind:\" \n",
      "\n",
      "en, and of \n",
      "chwp prom, the been ind mins in his pare whhich aws beron, but fror pamer prere seperon the to \n",
      "noy thim \n",
      "not the \n",
      "\"ther weragat the so dhere prre, in \n",
      "\n",
      "Whe wenew of the in teran the \n",
      "any bing a In lingh a The the win the wame it tas in sup- \n",
      "\n",
      "Ond ment. \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "WThe reed \n",
      "his diled of and ther and noplingus and \"gighor we in of nen of thes peawing lere the ding the ande reC4 \n",
      "Nofth deng siorfirice in the punged noun on \n",
      "tho sey old fro- \n",
      "earing sais \n",
      "dirove EAf what w \n",
      "\n",
      "[0m 51s (300 15%) 1.9856]\n",
      "Le's oo her end thik of to hen a thot sit, she the dome foldred and to ordied ass the lardacit! \n",
      "\n",
      "\n",
      "\"CAS whiech, able- Al. \n",
      "\n",
      "\"I mrived aPryod to the on mepech of had the what shout \n",
      "at him the gached milly of he carred the thostit to his memined to the \n",
      "sely coolived for the or nog the id the \n",
      "ass ande ricentithing it have of he thim of countrader oon hen, her ow that \n",
      "alwey sto of \n",
      "awd \n",
      "mranced the ount rotise beat and an touz, that the and the crivised hot the \n",
      "her ou the care ar, had tith the upul the bal said have all \n",
      "in of on to the wif she capt \n",
      "wenthed chited nothing the and not and the sokye all frtill the rerry brocess of of \n",
      "arom soul adone and the evillat sher and the ricerced ou culled dal to of is her \n",
      "caind and to by of cohom of bast thome \n",
      "whith ady of alil by of his for used this the \n",
      "the at of him, bles \n",
      "and to leant- the set dad on deal the the \n",
      "she? the and the you thin the \n",
      "fordercom and on whith doused the wit an the \n",
      "iped, the to cou chakis the was ald bet. \n",
      "math of \n",
      "\n",
      "[1m 9s (400 20%) 2.1250]\n",
      "Le!: \n",
      "\n",
      "\"Vwe comrowning, and sithed she the aid this in she the son natiher . noH for she \n",
      "sout in trages \n",
      "she?\" segah huce at into \n",
      "\n",
      "dre\" nowt hacis. \n",
      "\n",
      "\n",
      "\"Aollow his shought for \n",
      "\n",
      "he up a \n",
      "on this fell to \n",
      "serled the \n",
      "gerple, guled the and Booths st6v, Sithe, \n",
      "there hand he compean, ploanged \n",
      "ster youn corcomather sen abould ar rant the in the \n",
      "passing a slee wat \n",
      "stound. An han, ut and ree she seating \n",
      "to dellow his ext poaly hen the fiverer, hald \n",
      "price oid \n",
      "cureney, \"Son, low so hup ampered \n",
      "the fire whis plaslesaitn to had \n",
      "lost and \n",
      "to sou he she the and hoy and the \n",
      "listare, his his respoppight le aole stughed the sish she set hy pakoned lod sies paste lition, and \n",
      "and the cosussic his ponianed lod reeppinting his red cest prinepesss, and her ans hounded the \n",
      "recrtyger his seveer, is thess spertion we \n",
      "praspimted, shat sais on and dilk, lell \n",
      "he red the rivesting the light she cace speinged \n",
      "tild the not he sime had segher \n",
      "had knore it sppere. He she sight one of the cantown yur no \n",
      "\n",
      "[1m 26s (500 25%) 1.8640]\n",
      "Le vike \n",
      "and had haver. \n",
      "\n",
      "An the wapasher on wark and \n",
      "was fron, and hert monsed to that the amove the \n",
      "was and he domen thery that and himng and femor thing the yat hin ant ently an and froather \n",
      "propsmalonceme of the mayed think dade undentres, \n",
      "noung the liffon had Pince manged and enorman the \n",
      "semparred head wordat his \n",
      "his and that he sofd, Shereg. . . \n",
      "\n",
      "Hemand and sover and and \n",
      "able fut the ganing \n",
      "dor the drow to have \n",
      "and and him up \n",
      "at a had puint warain \n",
      "prpashy had \n",
      "theave many and her \n",
      "the Which he plea; band- \n",
      "it a mish, ham some was lome deld aring the \n",
      "wilitions a saygullars the \n",
      "armarp the call he of the pracpessed and suned \n",
      "her the \n",
      "tirmed ould you that not the sup withok have carrerlacain mustirsaints and and fare the \n",
      "ady parthed the rear, and have \n",
      "graped anntally \n",
      "wendesposorp the housty. \n",
      "\n",
      "\n",
      "\n",
      "\"Coring the ferom shout he and the ceplastally and \n",
      "and sher tand with livg the Thays and on, \n",
      "the the gandent the carts and to the sande and and thing the for, whas had and y \n",
      "\n",
      "[1m 42s (600 30%) 1.8295]\n",
      "Lers lout \n",
      "were bee their it hier says the pupide fule \n",
      "all the dich beest plinge, a yould impostion that wor the alon of would \n",
      "his, thoris anters. That or grangules \n",
      "me there \n",
      "it that was and Prang. \n",
      "\n",
      "The day ment stand a bal \n",
      "fathervers, at in and beguan the ment was and \n",
      "jup of toub a the maped of the cent or as to his in in drinc- \n",
      "otheris in Empere the becse as alan to has is for re to not to is look and the geting \n",
      "for aparing now he ebere his paly of to \n",
      "he to gan men \n",
      "inel \n",
      "agations and Fressill sebe the \n",
      "pastonciov \n",
      "mans, appose the ase or of to hously and to purarioos the lean beady and \n",
      "the alabom be did but there a craire. Sta- \n",
      "pours, and a sout pozombok to the Rostove isshe \n",
      "\n",
      "mass the brod hold \n",
      "all bight \n",
      "in a must win a fulver is was of lunt as in had to the \n",
      "wars many doperg the Trastivnats and \n",
      "and id to she actiling the mars no- \n",
      "in of to is wherounkaken some sill \n",
      "that was and, the hove beove yo. Song, molowan dove that \n",
      "D61lobely he therar balant had with tout that  \n",
      "\n",
      "[2m 0s (700 35%) 1.9406]\n",
      "Le is mayed, g-ple of I com all papts, not the \n",
      "intestoried that was and exan or the \n",
      "fembed he him stattrise and host and moveven round not with yout on wery solly. \n",
      "\n",
      "\"Kent bre- me seen my to him re-- \n",
      "riden knoded have and not tho the olf the yamy and but why an \n",
      "Krive ring her the coun the Anon bode not al mone han's his flack- \n",
      "tand and heen a yand, aws grow mord \n",
      "ramm not I noblover wenthabalot with that \n",
      "you fuplied in stat fole saw. We Annt at the elan got and \n",
      "with it exeplor my mobess of the fan since seen't to cicture \n",
      "this lit you of ever that the my evering with sat \n",
      "in't him my not his and \n",
      "conseld the conge he when mabor compal- \n",
      "re min fett, jage offaned the rance this \n",
      "my, I come and angrove it he ofle his feple net plove ban flon the \n",
      "atands maw sprinticuced hormed doich thorr himd, had fre- \n",
      "imep to he \n",
      "havile; \"han for some evrning to stited the manened \n",
      "his why as thoughancant to \n",
      "the ploven'm and him and but and let staid \n",
      "thim had looked ance with comper am is the w \n",
      "\n"
     ]
    }
   ],
   "source": [
    "#n_epochs = 2000\n",
    "n_epochs = 2000\n",
    "print_every = 100\n",
    "plot_every = 10\n",
    "hidden_size = 100\n",
    "n_layers = 1\n",
    "lr = 0.005\n",
    "\n",
    "decoder = RNN(n_characters, hidden_size, n_characters, n_layers)\n",
    "decoder_optimizer = torch.optim.Adam(decoder.parameters(), lr=lr)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "start = time.time()\n",
    "all_losses = [0]\n",
    "loss_avg = 0\n",
    "\n",
    "for epoch in range(1, n_epochs + 1):\n",
    "    #loss = train(*random_training_set())  \n",
    "    loss = train(*random_training_set())\n",
    "    loss_avg += loss\n",
    "\n",
    "    if epoch % print_every == 0:\n",
    "        print('[%s (%d %d%%) %.4f]' % (time_since(start), epoch, epoch / n_epochs * 100, loss))\n",
    "        print(evaluate('Le', 500), '\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
